{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_utils.basic_data import load_train_val_dataset_cross"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "split the dataset\n",
      "saved split 1\n",
      "saved split 2\n",
      "saved split 3\n",
      "saved split 4\n",
      "saved split 5\n",
      "saved split 6\n",
      "saved split 7\n",
      "saved split 8\n"
     ]
    }
   ],
   "source": [
    "testdf,traindf = load_train_val_dataset_cross(3,8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "625"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(testdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4381"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(traindf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>entity</th>\n",
       "      <th>negative</th>\n",
       "      <th>key_entity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4645</td>\n",
       "      <td>eadc641b</td>\n",
       "      <td>//@日式美睫美甲师小归:海象理财公司管管吧。不兑付～逾期，毁了多少人的生活和梦想</td>\n",
       "      <td>//@日式美睫美甲师小归:海象理财公司管管吧。不兑付～逾期，毁了多少人的生活和梦想</td>\n",
       "      <td>海象理财</td>\n",
       "      <td>1</td>\n",
       "      <td>海象理财</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4583</td>\n",
       "      <td>bfe5afde</td>\n",
       "      <td>我不需要你是个盖世英雄，京东白条提现~京东白条可以提现吗~京东白条提现也不希望你有举世无双的...</td>\n",
       "      <td>我不需要你是个盖世英雄，京东白条提现~京东白条可以提现吗~京东白条提现也不希望你有举世无双的...</td>\n",
       "      <td>京东白条</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4954</td>\n",
       "      <td>8e3a2403</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018年8月6日，央行开出天价罚单，国付宝、联动优势因为非法交易提供支付服务等，分别被罚约...</td>\n",
       "      <td>国付;国付宝;联动优势</td>\n",
       "      <td>1</td>\n",
       "      <td>国付宝;联动优势</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3402</td>\n",
       "      <td>9dbabf87</td>\n",
       "      <td>????《雷潮周年总结（上）：影响行情走向的几个关键点》唐小僧的爆雷，引发同为四大“高返”平...</td>\n",
       "      <td>????《雷潮周年总结（上）：影响行情走向的几个关键点》唐小僧的爆雷，引发同为四大高返平台的...</td>\n",
       "      <td>联璧金融;唐小僧</td>\n",
       "      <td>1</td>\n",
       "      <td>唐小僧;联璧金融</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1881</td>\n",
       "      <td>06d100b9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5、贵阳开发大数据防控金融风险系统平台，设置区块链系统等八大模块从贵阳市网络借贷信息中介机构...</td>\n",
       "      <td>借贷信息;小贷;小额贷;区块链系统</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>620</td>\n",
       "      <td>877</td>\n",
       "      <td>bd5936b6</td>\n",
       "      <td>如果有一京东白条提现一勇敢·互助·感恩—京东白条提现—对话宜宾地震灾区受灾群众#京东白条怎么...</td>\n",
       "      <td>如果有一京东白条提现一勇敢·互助·感恩—京东白条提现—对话宜宾地震灾区受灾群众#京东白条怎么...</td>\n",
       "      <td>京东白条</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>621</td>\n",
       "      <td>3783</td>\n",
       "      <td>9a9a3eea</td>\n",
       "      <td>????#银监会[超话]##小资钱包涉嫌诈骗[超话]#2019年4月26日，北京市公安局海淀...</td>\n",
       "      <td>????#银监会[超话]##小资钱包涉嫌诈骗[超话]#2019年4月26日，北京市公安局海淀...</td>\n",
       "      <td>资易贷（北京）金融信息服务有限公司;小资钱包;资易贷</td>\n",
       "      <td>1</td>\n",
       "      <td>资易贷（北京）金融信息服务有限公司;小资钱包</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>622</td>\n",
       "      <td>738</td>\n",
       "      <td>69b9a64b</td>\n",
       "      <td>????#宜贷网#6.5亿所谓错配资产就是承接摩尔龙的6.5亿坏账不良资产。多年截流和赚的钱...</td>\n",
       "      <td>????#宜贷网#6.5亿所谓错配资产就是承接摩尔龙的6.5亿坏账不良资产。多年截流和赚的钱...</td>\n",
       "      <td>北京金领贷;宜贷网(沪);摩尔龙;宜贷网</td>\n",
       "      <td>1</td>\n",
       "      <td>摩尔龙;宜贷网</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>623</td>\n",
       "      <td>3662</td>\n",
       "      <td>ead6fa36</td>\n",
       "      <td>NaN</td>\n",
       "      <td>116北京易商通科技股份有限公司非法吸收公众存款案</td>\n",
       "      <td>北京易商通科技股份有限公司;易商通;易商通科技;京易商通科技股份有限公司</td>\n",
       "      <td>1</td>\n",
       "      <td>京易商通科技股份有限公司</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>624</td>\n",
       "      <td>2530</td>\n",
       "      <td>7adf2ab5</td>\n",
       "      <td>重磅!中弘股份股票被深交所依法依规终止</td>\n",
       "      <td>重磅！中弘股份股票被深交所依法依规终止上市 2018年11月8日，根据《股票上市规则》规定...</td>\n",
       "      <td>中弘股份;深交所</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>625 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index        id                                              title  \\\n",
       "0     4645  eadc641b          //@日式美睫美甲师小归:海象理财公司管管吧。不兑付～逾期，毁了多少人的生活和梦想   \n",
       "1     4583  bfe5afde  我不需要你是个盖世英雄，京东白条提现~京东白条可以提现吗~京东白条提现也不希望你有举世无双的...   \n",
       "2     4954  8e3a2403                                                NaN   \n",
       "3     3402  9dbabf87  ????《雷潮周年总结（上）：影响行情走向的几个关键点》唐小僧的爆雷，引发同为四大“高返”平...   \n",
       "4     1881  06d100b9                                                NaN   \n",
       "..     ...       ...                                                ...   \n",
       "620    877  bd5936b6  如果有一京东白条提现一勇敢·互助·感恩—京东白条提现—对话宜宾地震灾区受灾群众#京东白条怎么...   \n",
       "621   3783  9a9a3eea  ????#银监会[超话]##小资钱包涉嫌诈骗[超话]#2019年4月26日，北京市公安局海淀...   \n",
       "622    738  69b9a64b  ????#宜贷网#6.5亿所谓错配资产就是承接摩尔龙的6.5亿坏账不良资产。多年截流和赚的钱...   \n",
       "623   3662  ead6fa36                                                NaN   \n",
       "624   2530  7adf2ab5                                重磅!中弘股份股票被深交所依法依规终止   \n",
       "\n",
       "                                                  text  \\\n",
       "0            //@日式美睫美甲师小归:海象理财公司管管吧。不兑付～逾期，毁了多少人的生活和梦想   \n",
       "1    我不需要你是个盖世英雄，京东白条提现~京东白条可以提现吗~京东白条提现也不希望你有举世无双的...   \n",
       "2    2018年8月6日，央行开出天价罚单，国付宝、联动优势因为非法交易提供支付服务等，分别被罚约...   \n",
       "3    ????《雷潮周年总结（上）：影响行情走向的几个关键点》唐小僧的爆雷，引发同为四大高返平台的...   \n",
       "4    5、贵阳开发大数据防控金融风险系统平台，设置区块链系统等八大模块从贵阳市网络借贷信息中介机构...   \n",
       "..                                                 ...   \n",
       "620  如果有一京东白条提现一勇敢·互助·感恩—京东白条提现—对话宜宾地震灾区受灾群众#京东白条怎么...   \n",
       "621  ????#银监会[超话]##小资钱包涉嫌诈骗[超话]#2019年4月26日，北京市公安局海淀...   \n",
       "622  ????#宜贷网#6.5亿所谓错配资产就是承接摩尔龙的6.5亿坏账不良资产。多年截流和赚的钱...   \n",
       "623                          116北京易商通科技股份有限公司非法吸收公众存款案   \n",
       "624   重磅！中弘股份股票被深交所依法依规终止上市 2018年11月8日，根据《股票上市规则》规定...   \n",
       "\n",
       "                                   entity  negative              key_entity  \n",
       "0                                    海象理财         1                    海象理财  \n",
       "1                                    京东白条         0                     NaN  \n",
       "2                             国付;国付宝;联动优势         1                国付宝;联动优势  \n",
       "3                                联璧金融;唐小僧         1                唐小僧;联璧金融  \n",
       "4                       借贷信息;小贷;小额贷;区块链系统         0                     NaN  \n",
       "..                                    ...       ...                     ...  \n",
       "620                                  京东白条         0                     NaN  \n",
       "621            资易贷（北京）金融信息服务有限公司;小资钱包;资易贷         1  资易贷（北京）金融信息服务有限公司;小资钱包  \n",
       "622                  北京金领贷;宜贷网(沪);摩尔龙;宜贷网         1                 摩尔龙;宜贷网  \n",
       "623  北京易商通科技股份有限公司;易商通;易商通科技;京易商通科技股份有限公司         1            京易商通科技股份有限公司  \n",
       "624                              中弘股份;深交所         0                     NaN  \n",
       "\n",
       "[625 rows x 7 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('tmp/train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 5])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.6048, grad_fn=<NllLossBackward>)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = nn.CrossEntropyLoss()\n",
    "input = torch.randn(3, 5, requires_grad=True)\n",
    "target = torch.empty(3, dtype=torch.long).random_(5)\n",
    "output = loss(input, target)\n",
    "output.backward()\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import reduce\n",
    "entity = reduce(lambda x,y:x+y,full['entity'].map(lambda x:str(x).split(';')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('tmp/stop_words_entity.txt','w') as f:\n",
    "    for e in entity:\n",
    "        f.write(e+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader,val_loader,token = get_train_val_data_loader(device=device,batch_size=16,shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "samples = [a for a in train_loader]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'plt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-74cb13e27d0c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'ABCD'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcumsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubplots\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlogy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'best'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'plt' is not defined"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "df =pd.DataFrame(np.random.rand(1000, 4), index=np.arange(1000), columns=list('ABCD'))   \n",
    "df = df.cumsum()  \n",
    "plt.figure()\n",
    "df.plot(subplots=True,logy=True)\n",
    "plt.legend(loc='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
